package com.jiuzhou.crawlers.taobao.pipeline.jd;

import java.util.Map;

import org.apache.commons.codec.digest.DigestUtils;
import org.apache.xerces.util.SynchronizedSymbolTable;

import com.jiuzhou.crawlers.taobao.util.HdfsUtil;

import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;

	/**
	 * 项目名称：Clawer
	 * @author yanganzhen
	 * version-code：1.0
     * 类说明：pipeline类-------爬取数据持久化类
     * 创建时间：2016年11月30日
	 */
public class JdGoodsPipeline implements Pipeline{
    private String filePath;
	public JdGoodsPipeline(String filePath){
		this.filePath =filePath;
	}
	
	public void process(ResultItems resultItems, Task arg1) {

		if (!this.filePath.endsWith("/"))
			this.filePath += "/";
	    System.out.println("-----------------------"+resultItems.getRequest().getUrl()+"---------------------------");
		String path = this.filePath + DigestUtils.md5Hex(resultItems.getRequest().getUrl());
		StringBuffer stringBuffer = new StringBuffer();
		stringBuffer.append("url:\t" + resultItems.getRequest().getUrl() + "\n");
		for (Map.Entry<String, Object> entry : resultItems.getAll().entrySet()) {
			    stringBuffer.append(entry.getKey() + ":\t" + entry.getValue() + "\n");
		}
		HdfsUtil.WriteToHdfs(path,stringBuffer.toString());
		// TODO Auto-generated method stub
	}

}
